<?php
include_once('../lib/config.php');
include_once('../lib/Db.class.php');
include_once('../lib/Model.class.php');
include_once('../lib/Snoopy.class.php');
include_once('../lib/function.php');
include_once('../lib/parseHtml.class.php');
include_once('/usr/local/xunsearch/sdk/php/lib/XS.php');
/**/
// from index_search.php
$redis = new Redis();  
$ret = $redis->connect("localhost","6379");  //php客户端设置的ip及端口
if(!$ret) die('redis connction faild');
$redis->auth(REDIS_PASS);
$redis->select(2);

$GLOBALS['config']['master']['db_database'] = 'gouyg';
$GLOBALS['config']['slave']['db_database']  = 'gouyg';

$xs = new XS('booksonline');
$index  = $xs->index;

$basedir = dirname(__FILE__);
$model = new Db;
$parseHtml = new parseHtml;
$limit = 100;
$use_proxy = true;
while ($limit--) {
    $queue_count = $redis->lSize('BOOK_CONTENT_QUEUE');
    if($queue_count > 10000) {
        echo2($limit . ', fetch queue over '.$queue_count);
        sleep(1);
        continue;
    }

    $q_string = $redis->lPop('BOOK_CHAPTER_QUEUE');
    if(!$q_string) {
        //echo2('BOOK_CHAPTER_QUEUE empty, limit:'.$limit);
        sleep(2);
        continue;
    }

    try {
        $data = json_decode($q_string, true);

        //获取url采集锁
        $locked_remove = ture;
        $locked_key = 'lockedbook:'.urlencode($data['url']);
        $locked = $redis->setNx($locked_key, time());
        if($locked) {
            $redis->expire($locked_key, 300);
        } else {
            $locked_remove = false;
            $q_string = null;
            throw new Exception('locked url:'.$data['url']);
        }

        $snoopy = new Snoopy;
        $snoopy->curl($data['url'], $use_proxy);

        if($snoopy->status == 0) {
            $use_proxy = !$use_proxy;
        }

        $parseResult = $parseHtml->getBook($data['url'], $snoopy->results);
        if(!$parseResult) {
            throw new Exception('该url地址没有相应解析');
        }

        list($last_update_time, $description, $matches, $parseurl) = $parseResult;
        if(count($matches[1]) == 0) {
            if($snoopy->status == 200) {
                $q_string = null;
            }
            throw new Exception('Status:'.$snoopy->status.', ErrorChapter Found，url:'.$data['url']);
        }

        //7天内没有更新从连载队列移除
        if($last_update_time < time() - 3600*24*7) {
            $redis->hDel('book_ongoing', $data['md5id']);
        }

        // 创建文档对象
        $data['description'] = $description;
        $data['urls_count'] = count($matches[1]);
        $data['updatetime'] = $last_update_time;
        $data['last_chapter'] = end($matches[2]);
        $data['last_chapter_url'] = getFullUrl(end($matches[1]), $data['url']);

        // 保存所有章节列表信息
        $redis->set('books:'.$data['md5id'], json_encode([
            'title'=>$data['title'],
            'last_chapter'=>$data['last_chapter'],
            'last_chapter_url'=>$data['last_chapter_url'],
            'addtime'=>time(),
        ]));

        // 存储book信息
        $old_book = $model->find('book', "md5id='".$data['md5id']."'");
        if(!$old_book) {
            $r = $model->add('book', $data);
            if(!$r) {
                $error = $model->error('book insert error');
                throw new Exception($error['info']);
            } else {
                $book_id = $model->insert_id();
            }
        } else {
            $book_id = $old_book['id'];
            if($old_book['updatetime'] < $last_update_time || $old_book['last_chapter'] != $data['last_chapter'] || $old_book['last_chapter_url'] != $data['last_chapter_url']) {
                $update_data = [
                    'updatetime'=>$last_update_time,
                    'last_chapter'=>$data['last_chapter'],
                    'last_chapter_url'=>$data['last_chapter_url'],
                ];
                if($description && empty($old_book['description'])) {
                    $update_data['description'] = $description;
                }

                $model->save('book', "id='{$book_id}'", $update_data);
            }
        }

        $doc = new XSDocument;
        $doc->setFields($data);

        // 更新到索引数据库中
        $index->update($doc);

        $model->startTrans();
        // 压入采集内容队列
        $old = $new = 0;
        foreach ($matches[1] as $k=>$url) {
            $url = getFullUrl($url, $data['url']);

            // 检查是否已下载
            $localfile = '/read_all/'.$data['md5id'].'/'.md5($url);
            $cachefile = $basedir . $localfile;

            // 存储chapter信息
            $old_chapter = $model->find('book_chapter', "md5url='".md5($url)."'");
            if(!$old_chapter) {
                $chapter_data = [
                    'book_id'=>$book_id,
                    'page_id'=>$k,
                    'md5url'=>md5($url),
                    'url'=>$url,
                    'title'=>$matches[2][$k],
                    'local_file'=>'',
                    'add_time'=>time(),
                ];
                $r = $model->add('book_chapter', $chapter_data);
                if(!$r) {
                    $error = $model->error('chapter insert error');
                    throw new Exception($error['info']);
                } else {
                    $chapter_id = $model->insert_id();
                }
                $new++;
            } else {
                $chapter_id = $old_chapter['id'];
                $old++;
            }

            if(!file_exists($cachefile)) {
                if($old_chapter && $old_chapter['remote_file'] == 1) {
                    echo2($localfile.' remote exists');
                } else {
                    $redis->rPush('BOOK_CONTENT_QUEUE', json_encode([$data['md5id'], $url, $chapter_id]));
                }
            }
                
            if( $old_chapter ) {
                $update_chapterdata = [];
                $update_chapterdata['page_id'] = $k;
                if(file_exists($cachefile) && empty($old_chapter['local_file'])) {
                    $update_chapterdata['local_file'] = $localfile;
                }

                $model->save('book_chapter', "id='{$chapter_id}'", $update_chapterdata);
            }    
        }

        $model->commit();

        //释放锁
        $redis->delete($locked_key);

        echo2('old:'.$old.', new:'.$new.', last:'.date('Y-m-d',$last_update_time).', 章节数:'.$data['urls_count'].', '.$data['title']);

    } catch (Exception $e) {
        echo2($e->getMessage());

        if($q_string) {
            $redis->rPush('BOOK_CHAPTER_QUEUE', $q_string);
        }

        if($locked_remove) {
            $redis->delete($locked_key);
        }

        $model->rollback();
        $model->end();
        sleep(2);
    }
}

echo2('BOOK_CHAPTER_QUEUE empty, limit:'.$limit);
